Benchmark pipeline

Number of proteins

Number of DEA results when also Proteins with single peptide.

## [1] "SPECNAUT_default"

Including single peptides

benchmark1$smc$summary
## # A tibble: 1 × 2
##   nr_missing protein_Id
##        <int>      <int>
## 1          0       8714
benchmark1$plot_scatter()

benchmark1$plot_score_distribution()
## Picking joint bandwidth of 0.05
## Picking joint bandwidth of 0.05
## Picking joint bandwidth of 2.38
## Picking joint bandwidth of 0.0392

benchmark1$plot_ROC(xlim = 0.3)
## Warning: Removed 12492 rows containing missing values (`geom_path()`).

benchmark1$pAUC_summaries()
## $barp

## 
## $ftable
## $ftable$content
## # A tibble: 6 × 5
## # Groups:   contrast [2]
##   contrast what             AUC pAUC_10 pAUC_20
##   <chr>    <chr>          <dbl>   <dbl>   <dbl>
## 1 B_vs_A   diff            98.9    94.0    96.6
## 2 B_vs_A   scaled.p.value  98.9    93.0    95.9
## 3 B_vs_A   statistic       98.9    93.2    96.0
## 4 all      diff            98.9    94.0    96.6
## 5 all      scaled.p.value  98.9    93.0    95.9
## 6 all      statistic       98.9    93.2    96.0
## 
## $ftable$caption
## [1] "AUC, and pAUC at 0.1 and 0.2 FPR for  (NC) protein level measurments, linear model"
## 
## $ftable$digits
## [1] 2
benchmark1$plot_precision_recall(precision_lim = 0)

At least two peptides

benchmark2$smc$summary
## # A tibble: 1 × 2
##   nr_missing protein_Id
##        <int>      <int>
## 1          0       7613
benchmark2$plot_scatter()

benchmark2$plot_score_distribution()
## Picking joint bandwidth of 0.0464
## Picking joint bandwidth of 0.0464
## Picking joint bandwidth of 2.58
## Picking joint bandwidth of 0.0399

benchmark2$plot_ROC(xlim = 0.3)
## Warning: Removed 10971 rows containing missing values (`geom_path()`).

benchmark2$pAUC_summaries()
## $barp

## 
## $ftable
## $ftable$content
## # A tibble: 6 × 5
## # Groups:   contrast [2]
##   contrast what             AUC pAUC_10 pAUC_20
##   <chr>    <chr>          <dbl>   <dbl>   <dbl>
## 1 B_vs_A   diff            99.7    98.6    99.2
## 2 B_vs_A   scaled.p.value  99.6    97.2    98.3
## 3 B_vs_A   statistic       99.6    97.2    98.4
## 4 all      diff            99.7    98.6    99.2
## 5 all      scaled.p.value  99.6    97.2    98.3
## 6 all      statistic       99.6    97.2    98.4
## 
## $ftable$caption
## [1] "AUC, and pAUC at 0.1 and 0.2 FPR for  (NC) protein level measurments, linear model"
## 
## $ftable$digits
## [1] 2
benchmark2$plot_precision_recall(precision_lim = 0)

Comparing different Software

Nr proteins

nr1 <- nr |> filter(nrPep == 1)
nr2 <- nr |> filter(nrPep == 2)
nr1 |> ggplot(aes(x = name, y = protein_Id)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

nr2 <- nr |> filter(nrPep == 2)
nr2 |> ggplot(aes(x = name, y = protein_Id)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

Nr peptides

Including single peptides

nr1 |> ggplot(aes(x = name, y = nrPeptides)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

Two or more peptides

nr2 |> ggplot(aes(x = name, y = nrPeptides)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

ROC areas 10% - p.value

Including single peptides

rocs <- bind_rows(lapply(all_RES, function(x){x$roc}))
rocs1 <- filter(rocs, nrPep == 1, what == "scaled.p.value", contrast == "B_vs_A")
rocs1 |> ggplot(aes(x = name, y = pAUC_10)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

Two or more peptides

rocs2 <- filter(rocs, nrPep == 2, what == "scaled.p.value", contrast == "B_vs_A")
rocs2 |> ggplot(aes(x = name, y = pAUC_10)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

ROC areas 10% - difference

Including single peptides

rocs1 <- filter(rocs, nrPep == 1, what == "diff", contrast == "B_vs_A")
rocs1 |> ggplot(aes(x = name, y = pAUC_10)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

two or more peptides

rocs2 <- filter(rocs, nrPep == 2, what == "diff", contrast == "B_vs_A")
rocs2 |> ggplot(aes(x = name, y = pAUC_10)) + geom_bar(stat="identity") +
   theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1))

Difference ranges

Look at standard deviations

##  [1] "DIANN_314_relaxed"           "DIANN_314"                   "DIANN_314_staggered_relaxed" "DIANN_314_staggered"         "DIANN_FP_19_1"              
##  [6] "DIANN_FP_19_1_staggered"     "DIANN_FP_312"                "DIANN_FP_312_staggered"      "SPECNAUT_default_Staggered"  "SPECNAUT_default"           
## [11] "SPECNAUT_NoImpute_Staggered"